fsdb 0.7.1 → 0.7.2

data/History.txt

@@ -1,3 +1,15 @@
+fsdb 0.7.2
+
+- rename lock_shared and lock_exclusive by appending _fsdb to avoid conflicts
+- require thread for 1.8.7 compat
+- added fixture example
+- fetch clears cache entry to prevent increasing memory use
+- fix possible race condition in insert
+- use db cache_mutex instead of Thread.exclusive.
+- minor api doc changes
+- stop using literal character notation, for clarity
+- updated README with recent benchmarks
+
 fsdb 0.7.1
 
 - added a require needed for ruby 1.8

data/README.markdown

@@ -291,17 +291,17 @@ FSDB is not very fast. It's useful more for its safety, flexibility, and ease of
   or FSDB dump/load methods that use a faster format (e.g., plain text, rather
   than a marshalled String), this may not be so bad.
 
-- On an 850MHz PIII under linux, with debugging turned off (-b option),
+- On an 1.3GHz Core 2 Duo under linux, with debugging turned off (-b option),
   test-concurrency.rb reports:
   
     processes | threads   | objects   | transactions per cpu second
     --------- | --------- | --------- | ---------------------------
-    1         | 1         | 10        | 965
-    1         | 10        | 10        | 165
-    10        | 1         | 10        | 684
-    10        | 10        | 10        | 122
-    10        | 10        | 100       | 100
-    10        | 10        | 10000     | 92
+    1         | 1         | 10        | 4798
+    1         | 10        | 10        | 3537
+    10        | 1         | 10        | 4231
+    10        | 10        | 10        | 4093
+    10        | 10        | 100       | 4060
+    10        | 10        | 10000     | 3700
 
     These results are not representative of typical applications, because the
   test was designed to stress the database and expose stability problems, not
@@ -521,22 +521,6 @@ is:
 
 ### Performance
 
-- Profiling says that Thread.exclusive consumes about 20% of cpu. Also,
-  Thread.stop and Thread.run when there are multiple threads. Using
-  Thread.critical in places where it is safe to do so (no exceptions raised)
-  instead of Thread.exclusive would reduce this to an estimated 6%.
-  ((See faster-modex .rb and faster-mutex.rb.))
-
-- Better way of waiting for file lock in the multithread case
-  
-  - this may be unfixable until ruby has native threads
-  
-- Use shared memory for the cache, so write is not necessary after edit.
-
-  - actually, this may not make much sense
-
-- Option for Database to ignore file locking and possibility of other writers.
-
 - fetch could use the cache better if the cache kept the file contents string
   as well as the loaded object. Then the #stale! call would only have to
   wipe the reference to the object, and could leave the contents string. But

data/examples/fixture.rb

@@ -0,0 +1,45 @@
+require 'fsdb'
+require 'tmpdir'
+
+data = FSDB::Database.new "data"
+
+formats = [
+  FSDB::TEXT_FORMAT.when(/\.txt$|\.html$/),
+  FSDB::BINARY_FORMAT.when(/\.png$/)
+]
+
+data.formats = formats
+
+Dir.mktmpdir do |dir|
+  #dir = "tmp1" # for testing, use a dir that won't be deleted
+  
+  tmp = FSDB::Database.new(dir)
+  tmp.formats = formats
+  
+  tmp["hello.txt"] = "contents of hello.txt"
+  tmp["web/snippet.html"] = "<h1>Fix this!</h1>"
+  
+  img = tmp.subdb("img")
+  img["hello.png"] = data["hello.png"]
+  img["hi.png"] = data["hello.png"]
+end
+
+# you can automatically serialize objects in yaml, json, or marshal:
+require 'yaml'
+require 'json'
+
+Dir.mktmpdir do |dir|
+  #dir = "tmp2" # for testing, use a dir that won't be deleted
+
+  tmp = FSDB::Database.new(dir)
+  json_format = FSDB::Format.new(
+    /\.json$/, /\.js$/,
+    :name => "JSON_FORMAT",
+    :load => proc {|f| JSON.load(f)},
+    :dump => proc {|object, f| f.syswrite(JSON.dump(object))}
+  )
+  tmp.formats = formats + [FSDB::YAML_FORMAT, json_format]
+  
+  tmp["a.yaml"] = {:some => ["complex", Object]}
+  tmp["b.json"] = ["foo", 2, 3]
+end

data/lib/fsdb/database.rb

@@ -85,12 +85,14 @@ class Database
       @object     = object
     end
     
-    def start_using; Thread.exclusive {@users += 1}; end
-    def stop_using;  Thread.exclusive {@users -= 1}; end
+    # called in context of lock on db's cache_mutex.
+    def start_using;  @users += 1; end
 
-    # called in context of lock on db's cache_mutex, which is also
-    # required for start_using.
-    def unused?;    @users == 0; end
+    # called in context of lock on db's cache_mutex.
+    def stop_using;   @users -= 1; end
+
+    # called in context of lock on db's cache_mutex.
+    def unused?;      @users == 0; end
     
     # Protects object during #browse, #edit, and so on. Should be locked
     # as long as the object is being used. It's ok to lock the @mutex
@@ -160,6 +162,8 @@ class Database
   
   # Create a new database object that accesses +path+ relative to the database
   # directory. A process can have any number of dbs accessing overlapping dirs.
+  # The FSDB concurrency protections apply to a file regardless of which db
+  # is used to access it.
   # The cost of creating an additional db is very low; its state is just the
   # dir and some options. Caching is done in structures owned by the Database
   # class itself.
@@ -174,10 +178,11 @@ class Database
   def inspect; "#<#{self.class}:#{dir}>"; end
   
   # Convert a relative path (relative to the db dir) to an absolute path.
+  # A directory path will have '/' appended to it.
   def absolute(path)
     abs_path = File.expand_path(File.join(@dir, path))
     if File.directory?(abs_path)
-      abs_path << ?/ # prevent Errno::EINVAL on UFS
+      abs_path << "/" # prevent Errno::EINVAL on UFS
     end
     abs_path
   end
@@ -346,7 +351,11 @@ private
     end
     yield cache_entry
   ensure
-    cache_entry.stop_using if cache_entry
+    if cache_entry
+      cache_mutex.synchronize do
+        cache_entry.stop_using
+      end
+    end
   end
 
   # Lock path for shared (read) use. Other threads will wait to modify it.
@@ -376,7 +385,7 @@ private
     rescue Errno::EINTR
       retry
     else
-      f.lock_shared(@lock_type)
+      f.lock_shared_fsdb(@lock_type)
       identify_file_type(f, path, abs_path)
       yield f
     ensure
@@ -398,7 +407,7 @@ private
     rescue Errno::EINTR
       retry
     else
-      f.lock_exclusive(@lock_type)
+      f.lock_exclusive_fsdb(@lock_type)
       identify_file_type(f, path, abs_path)
       yield f
     ensure
@@ -493,7 +502,7 @@ public
       end
 
       cache_entry.file_handle = f
-      f.lock_shared(@lock_type)
+      f.lock_shared_fsdb(@lock_type)
       identify_file_type(f, path, abs_path)
         ## could avoid if cache_object says so
       object = cache_object(f, cache_entry)
@@ -615,18 +624,14 @@ public
   # Insert the object, replacing anything at the path. Returns the object.
   # (The object remains a <i>local copy</i>, distinct from the one which will be
   # returned when accessing the path through database transactions.)
-  #
-  # If +path+ ends in "/", then object is treated as a collection of key-value
-  # pairs, and each value is inserted at the corresponding key under +path+.
-  # (You can omit the "/" if the dir already exists.)
-  ### is this still true?
   def insert(path, object)
     abs_path = absolute(path)
     file_id = make_file_id(abs_path)
     object_exclusive file_id do |cache_entry|
       open_write_lock(path) do |f|
         dump(object, f)
-        cache_entry.update(f.mtime, inc_version_of(f, cache_entry), object)
+        inc_version_of(f, cache_entry)
+        cache_entry.stale!
         object
       end
     end
@@ -647,7 +652,7 @@ public
   rescue MissingFileError
     raise DirIsImmutableError if PLATFORM_IS_WINDOWS_ME
   ensure
-    clear_entry(file_id) # no one else can get this copy of object
+    clear_entry(file_id) # the entry was recently marked stale anyway
   end
   alias []= insert
   
@@ -750,6 +755,8 @@ public
     raise unless File.directory?(abs_path)
     # on some platforms, opening a dir raises EACCESS
     return Formats::DIR_LOAD_FROM_PATH[abs_path]
+  ensure
+    clear_entry(file_id) # the entry was recently marked stale anyway
   end
   alias [] fetch
   

data/lib/fsdb/file-lock.rb

@@ -10,17 +10,17 @@ class File
   if FSDB::PLATFORM_IS_WINDOWS_ME
     # no flock() on WinME
 
-    def lock_exclusive lock_type    # :nodoc
+    def lock_exclusive_fsdb lock_type    # :nodoc
     end
     
-    def lock_shared lock_type   # :nodoc
+    def lock_shared_fsdb lock_type   # :nodoc
     end
   
   else
     # Get an exclusive (i.e., write) lock on the file.
     # If the lock is not available, wait for it without blocking other ruby
     # threads.
-    def lock_exclusive lock_type
+    def lock_exclusive_fsdb lock_type
       send(lock_type, LOCK_EX)
     rescue Errno::EINTR
       retry
@@ -29,7 +29,7 @@ class File
     # Get a shared (i.e., read) lock on the file.
     # If the lock is not available, wait for it without blocking other ruby
     # threads.
-    def lock_shared lock_type
+    def lock_shared_fsdb lock_type
       send(lock_type, LOCK_SH)
     rescue Errno::EINTR
       retry

data/lib/fsdb/modex.rb

@@ -1,3 +1,5 @@
+require 'thread'
+
 module FSDB
 
 # Modex is a modal exclusion semaphore.

data/lib/fsdb/persistent.rb

@@ -27,7 +27,7 @@ module Persistent
     persistent_mutex.synchronize do
       File.makedirs(File.dirname(persistent_file))
       File.open(persistent_file, "wb") do |f|
-        f.lock_exclusive do
+        f.lock_exclusive_fsdb do
           dump(f)
           yield self if block_given?
         end
@@ -50,7 +50,7 @@ module Persistent
     # a particular object, or there will be multiple copies.
     def restore file
       object = File.open(file, "rb") do |f|
-        f.lock_shared do
+        f.lock_shared_fsdb do
           load(f)
         end
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fsdb
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2011-12-06 00:00:00.000000000 Z
+date: 2012-05-20 00:00:00.000000000 Z
 dependencies: []
 description: ! 'A file system data base. Provides a thread-safe, process-safe Database
   class.
@@ -30,6 +30,8 @@ files:
 - README.markdown
 - bench/bench.rb
 - examples/indexes.rb
+- examples/fixture.rb
+- examples/data/hello.png
 - examples/rbformat.rb
 - examples/formats.rb
 - examples/yaml.rb
@@ -88,8 +90,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: fsdb
-rubygems_version: 1.8.11
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: File System Database
 test_files: []
+has_rdoc: